{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0511.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/0511.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WAKyo1d-DX6y",
        "colab_type": "text"
      },
      "source": [
        "## 10.6 求近义词和类比词"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sxi91gTca-eI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install mxnet-cu100 d2lzh"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5SoSf8GX62rU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9d307dd0-d62f-479a-8f99-e45caaa6c519"
      },
      "source": [
        "!cat /usr/local/cuda/version.txt"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA Version 10.0.130\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KpupiQ9m69Cd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "48567a9b-139c-4882-d468-d05d19e67aa1"
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.contrib import text \n",
        "\n",
        "text.embedding.get_pretrained_file_names().keys()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['glove', 'fasttext'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qOkfuA5i-CAo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "fa4201cf-c103-4b08-c45c-29b8e76261cf"
      },
      "source": [
        "print(text.embedding.get_pretrained_file_names('glove'))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['glove.42B.300d.txt', 'glove.6B.50d.txt', 'glove.6B.100d.txt', 'glove.6B.200d.txt', 'glove.6B.300d.txt', 'glove.840B.300d.txt', 'glove.twitter.27B.25d.txt', 'glove.twitter.27B.50d.txt', 'glove.twitter.27B.100d.txt', 'glove.twitter.27B.200d.txt']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8RZRekx--MeF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "c944873d-23a4-4a00-e4ee-70b59d1dd60b"
      },
      "source": [
        "glove_6b50d = text.embedding.create('glove', pretrained_file_name='glove.6B.50d.txt')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading /root/.mxnet/embeddings/glove/glove.6B.zip from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/embeddings/glove/glove.6B.zip...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hh3DejbG-zQk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0c780227-3ca3-45e2-bfb7-cc426d8f41a5"
      },
      "source": [
        "len(glove_6b50d)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "400001"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHA6Ch1P_Cj6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fe1a9e5e-f852-4d06-b3f9-a2fcaa01c63c"
      },
      "source": [
        "glove_6b50d.token_to_idx['beautiful']"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3367"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C36a6ojd_M4I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "48122310-d91f-4ce0-ae37-af134c8033a4"
      },
      "source": [
        "glove_6b50d.idx_to_token[3367]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'beautiful'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lASkaQv4_SLw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def knn(W, x, k):\n",
        "    cos = nd.dot(W, x.reshape((-1,))) / ((nd.sum(W * W, axis=1) + 1e-9).sqrt() * nd.sum(x * x).sqrt())\n",
        "    topk = nd.topk(cos, k=k, ret_typ='indices').asnumpy().astype('int32')\n",
        "    return topk, [cos[i].asscalar() for i in topk]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqVahHxiA1kp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_similar_tokens(query_token, k, embed):\n",
        "    topk, cos = knn(embed.idx_to_vec, embed.get_vecs_by_tokens([query_token]), k+1)\n",
        "    for i, c in zip(topk[1:], cos[1:]):\n",
        "        print('cosine sim=%.3f: %s' % (c, (embed.idx_to_token[i])))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bPekUGLUBsVo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "d4984f99-5f8f-4c19-e99d-68658d24e692"
      },
      "source": [
        "get_similar_tokens('chip', 9, glove_6b50d)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cosine sim=0.856: chips\n",
            "cosine sim=0.749: intel\n",
            "cosine sim=0.749: electronics\n",
            "cosine sim=0.731: semiconductor\n",
            "cosine sim=0.716: maker\n",
            "cosine sim=0.708: computer\n",
            "cosine sim=0.707: microprocessor\n",
            "cosine sim=0.703: makers\n",
            "cosine sim=0.701: micro\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ySogOeE2ByJH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "343b6ae6-c1f4-476c-c2c6-c14947318e4d"
      },
      "source": [
        "get_similar_tokens('baby', 9, glove_6b50d)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cosine sim=0.839: babies\n",
            "cosine sim=0.800: boy\n",
            "cosine sim=0.792: girl\n",
            "cosine sim=0.778: newborn\n",
            "cosine sim=0.765: pregnant\n",
            "cosine sim=0.762: mom\n",
            "cosine sim=0.759: child\n",
            "cosine sim=0.756: toddler\n",
            "cosine sim=0.750: mother\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMI0xwLXCHQ0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "b0b7c231-5d2b-4492-e3bd-73c37a743ddc"
      },
      "source": [
        "get_similar_tokens('beautiful', 9, glove_6b50d)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cosine sim=0.921: lovely\n",
            "cosine sim=0.893: gorgeous\n",
            "cosine sim=0.830: wonderful\n",
            "cosine sim=0.825: charming\n",
            "cosine sim=0.801: beauty\n",
            "cosine sim=0.774: elegant\n",
            "cosine sim=0.758: looks\n",
            "cosine sim=0.736: love\n",
            "cosine sim=0.735: graceful\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjpU5oTuCNgi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_analogy(token_a, token_b, token_c, embed):\n",
        "    vecs = embed.get_vecs_by_tokens([token_a, token_b, token_c])\n",
        "    x = vecs[1] - vecs[0] + vecs[2]\n",
        "    topk, cos = knn(embed.idx_to_vec, x, 1)\n",
        "    return embed.idx_to_token[topk[0]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Ql-oa2iDVni",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f9361c54-31e8-4545-e4f0-99214a4a64f2"
      },
      "source": [
        "get_analogy('man', 'woman', 'son', glove_6b50d)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'daughter'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6x5QqyQ4DfpI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ee0f6308-0fc9-4c4f-b966-ca73b5325ce5"
      },
      "source": [
        "get_analogy('beijing', 'china', 'tokyo', glove_6b50d)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'japan'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bNbBSVGLDqFP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8afeaa55-b168-4337-f5a6-4bf0dc11708f"
      },
      "source": [
        "get_analogy('bad', 'worst', 'big', glove_6b50d)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'biggest'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2l6H2GPJD1T-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f7660c08-e3cf-480c-8950-036f41bdf74f"
      },
      "source": [
        "get_analogy('do', 'did', 'go', glove_6b50d)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'went'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nwHx7tRrD882",
        "colab_type": "text"
      },
      "source": [
        "## 10.7 文本情感分类：使用循环神经网络"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhNJ-gxZET-P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import collections \n",
        "import d2lzh as d2l \n",
        "from mxnet import gluon, init, nd \n",
        "from mxnet.contrib import text \n",
        "from mxnet.gluon import data as gdata, loss as gloss, nn, rnn, utils as gutils \n",
        "import os \n",
        "import random \n",
        "import tarfile "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLl9Z91VGvor",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir -p ../data/aclImdb"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fh_BXbzVFRrI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "25a6bd16-4dc2-4780-bc86-81a2ec057e87"
      },
      "source": [
        "def download_imdb(data_dir='../data'):\n",
        "    url = ('http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz')\n",
        "    sha1 = '01ada507287d82875905620988597833ad4e0903'\n",
        "    fname = gutils.download(url, data_dir, sha1_hash=sha1)\n",
        "    with tarfile.open(fname, 'r') as f:\n",
        "        f.extractall(data_dir)\n",
        "\n",
        "download_imdb()"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading ../data/aclImdb_v1.tar.gz from http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Csd_Vob-Gptw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_imdb(folder='train'):\n",
        "    data = []\n",
        "    for label in ['pos', 'neg']:\n",
        "        folder_name = os.path.join('../data/aclImdb/', folder, label)\n",
        "        for file in os.listdir(folder_name):\n",
        "            with open(os.path.join(folder_name, file), 'rb') as f:\n",
        "                review = f.read().decode('utf-8').replace('\\n', '').lower()\n",
        "                data.append([review, 1 if label == 'pos' else 0])\n",
        "    random.shuffle(data)\n",
        "    return data \n",
        "\n",
        "train_data, test_data = read_imdb('train'), read_imdb('test')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GOxPFM_Liy-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_tokenized_imdb(data):\n",
        "    def tokenizer(text):\n",
        "        return [tok.lower() for tok in text.split(' ')]\n",
        "    return [tokenizer(review) for review, _ in data]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGmF9ynnMCM7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4b3a2f67-6b9a-4874-cc1b-d7c50e626b31"
      },
      "source": [
        "def get_vocab_imdb(data):\n",
        "    tokenized_data = get_tokenized_imdb(data)\n",
        "    counter = collections.Counter([tk for st in tokenized_data for tk in st])\n",
        "    return text.vocab.Vocabulary(counter, min_freq=5)\n",
        "\n",
        "vocab = get_vocab_imdb(train_data)\n",
        "'# words in vocab:', len(vocab)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('# words in vocab:', 46151)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vavasZyUNBZD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess_imdb(data, vocab):\n",
        "    max_l = 500 \n",
        "\n",
        "    def pad(x):\n",
        "        return x[:max_l] if len(x) > max_l else x + [0] * (max_l - len(x))\n",
        "    \n",
        "    tokenized_data = get_tokenized_imdb(data)\n",
        "    features = nd.array([pad(vocab.to_indices(x)) for x in tokenized_data])\n",
        "    labels = nd.array([score for _, score in data])\n",
        "    return features, labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IMVxYcaGO3R8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64 \n",
        "train_set = gdata.ArrayDataset(*preprocess_imdb(train_data, vocab))\n",
        "test_set = gdata.ArrayDataset(*preprocess_imdb(test_data, vocab))\n",
        "train_iter = gdata.DataLoader(train_set, batch_size, shuffle=True)\n",
        "test_iter = gdata.DataLoader(test_set, batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pw_TV4itQQcw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "2936639d-f684-4ae9-ba96-4182bf197e78"
      },
      "source": [
        "for X, y in train_iter:\n",
        "    print('X', X.shape, 'y', y.shape)\n",
        "    break \n",
        "'#batches:', len(train_iter)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X (64, 500) y (64,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('#batches:', 391)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "alLqcUuSQgTQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BiRNN(nn.Block):\n",
        "    def __init__(self, vocab, embed_size, num_hiddens, num_layers, **kwargs):\n",
        "        super(BiRNN, self).__init__(**kwargs)\n",
        "        self.embedding = nn.Embedding(len(vocab), embed_size)\n",
        "        self.encoder = rnn.LSTM(num_hiddens, num_layers=num_layers, bidirectional=True, input_size=embed_size)\n",
        "        self.decoder = nn.Dense(2)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeddings = self.embedding(inputs.T)\n",
        "        outputs = self.encoder(embeddings)\n",
        "        encoding = nd.concat(outputs[0], outputs[-1])\n",
        "        outs = self.decoder(encoding)\n",
        "        return outs "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p3M9t8IDS5Sd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embed_size, num_hiddens, num_layers, ctx = 100, 100, 2, d2l.try_all_gpus()\n",
        "net = BiRNN(vocab, embed_size, num_hiddens, num_layers)\n",
        "net.initialize(init.Xavier(), ctx=ctx)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3U2XEF4vTNBK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "glove_embedding = text.embedding.create('glove', pretrained_file_name='glove.6B.100d.txt', vocabulary=vocab)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T-nLDNjRTnI4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net.embedding.weight.set_data(glove_embedding.idx_to_vec)\n",
        "net.embedding.collect_params().setattr('grad_req', 'null')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RzK34STKT_vp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "f4f6fad2-638f-422b-d050-ffbbc9fe8ed6"
      },
      "source": [
        "lr, num_epochs = 0.01, 5 \n",
        "trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr})\n",
        "loss = gloss.SoftmaxCrossEntropyLoss()\n",
        "d2l.train(train_iter, test_iter, net, loss, trainer, ctx, num_epochs)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training on [gpu(0)]\n",
            "epoch 1, loss 0.5538, train acc 0.704, test acc 0.811, time 45.7 sec\n",
            "epoch 2, loss 0.4081, train acc 0.822, test acc 0.835, time 45.5 sec\n",
            "epoch 3, loss 0.3668, train acc 0.844, test acc 0.812, time 45.1 sec\n",
            "epoch 4, loss 0.3333, train acc 0.858, test acc 0.846, time 45.3 sec\n",
            "epoch 5, loss 0.2922, train acc 0.878, test acc 0.847, time 45.3 sec\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rb4_VVKjUusR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict_sentiment(net, vocab, sentence):\n",
        "    sentence = nd.array(vocab.to_indices(sentence), ctx=d2l.try_gpu())\n",
        "    label = nd.argmax(net(sentence.reshape((1, -1))), axis=1)\n",
        "    return 'positive' if label.asscalar() == 1 else 'negative'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFkVOZALWfzB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "21b20894-10f2-42bc-8f49-718ebd3dd949"
      },
      "source": [
        "predict_sentiment(net, vocab, ['this', 'movie', 'is', 'so', 'great'])"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'positive'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ug517jMVW0lT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "336cc854-c5f8-47a5-b62b-82818269d20c"
      },
      "source": [
        "predict_sentiment(net, vocab, ['this', 'movie', 'is', 'so', 'bad'])"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'negative'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_TgwFdSW6yB",
        "colab_type": "text"
      },
      "source": [
        "## 10.8 文本情感分类：使用卷积神经网络 ( textCNN )"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vct4--RyYTfV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import d2lzh as d2l \n",
        "from mxnet import gluon, init, nd \n",
        "from mxnet.contrib import text \n",
        "from mxnet.gluon import data as gdata, loss as gloss, nn "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrmasmMaYnmd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def corr1d(X, K):\n",
        "    w = K.shape[0]\n",
        "    Y = nd.zeros((X.shape[0] - w + 1))\n",
        "    for i in range(Y.shape[0]):\n",
        "        Y[i] = (X[i: i + w] * K).sum()\n",
        "    return Y "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lug8_2umZX5k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "f2cf460e-8768-4393-f8e6-77cce16188cc"
      },
      "source": [
        "X, K = nd.array([0, 1, 2, 3, 4, 5, 6]), nd.array([1, 2])\n",
        "corr1d(X, K)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[ 2.  5.  8. 11. 14. 17.]\n",
              "<NDArray 6 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qyd0X102ZofS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "30f32424-f9ff-47b0-e4de-d34666bfdc9f"
      },
      "source": [
        "def corr1d_multi_in(X, K):\n",
        "    return nd.add_n(*[corr1d(x, k) for x, k in zip(X, K)])\n",
        "\n",
        "X = nd.array([[0, 1, 2, 3, 4, 5, 6], \n",
        "        [1, 2, 3, 4, 5, 6, 7], \n",
        "        [2, 3, 4, 5, 6, 7, 8]])\n",
        "K = nd.array([[1, 2], [3, 4], [-1, -3]])\n",
        "corr1d_multi_in(X, K)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[ 2.  8. 14. 20. 26. 32.]\n",
              "<NDArray 6 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sWW2fNs6bFI2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64 \n",
        "d2l.download_imdb()\n",
        "train_data, test_data = d2l.read_imdb('train'), d2l.read_imdb('test')\n",
        "vocab = d2l.get_vocab_imdb(train_data)\n",
        "train_iter = gdata.DataLoader(gdata.ArrayDataset(*d2l.preprocess_imdb(train_data, vocab)), batch_size, shuffle=True)\n",
        "test_iter = gdata.DataLoader(gdata.ArrayDataset(*d2l.preprocess_imdb(test_data, vocab)), batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2j_xqgh5c82D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TextCNN(nn.Block):\n",
        "    def __init__(self, vocab, embed_size, kernel_size, num_channels, **kwargs):\n",
        "        super(TextCNN, self).__init__(**kwargs)\n",
        "        self.embedding = nn.Embedding(len(vocab), embed_size)\n",
        "        self.constant_embedding = nn.Embedding(len(vocab), embed_size)\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        self.decoder = nn.Dense(2)\n",
        "        self.pool = nn.GlobalMaxPool1D()\n",
        "        self.convs = nn.Sequential()\n",
        "        for c, k in zip(num_channels, kernel_sizes):\n",
        "            self.convs.add(nn.Conv1D(c, k, activation='relu'))\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeddings = nd.concat(self.embedding(inputs), self.constant_embedding(inputs), dim=2)\n",
        "        embeddings = embeddings.transpose((0, 2, 1))\n",
        "        encoding = nd.concat(*[nd.flatten(self.pool(conv(embeddings))) for conv in self.convs], dim=1)\n",
        "        outputs = self.decoder(self.dropout(encoding))\n",
        "        return outputs "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p3lL3y19f9GL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embed_size, kernel_sizes, nums_channels = 100, [3, 4, 5], [100, 100, 100]\n",
        "ctx = d2l.try_all_gpus()\n",
        "net = TextCNN(vocab, embed_size, kernel_sizes, nums_channels)\n",
        "net.initialize(init.Xavier(), ctx=ctx)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QQrlReAgbu6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "glove_embedding = text.embedding.create('glove', pretrained_file_name='glove.6B.100d.txt', vocabulary=vocab) \n",
        "net.embedding.weight.set_data(glove_embedding.idx_to_vec)\n",
        "net.constant_embedding.weight.set_data(glove_embedding.idx_to_vec)\n",
        "net.constant_embedding.collect_params().setattr('grad_req', 'null')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UcHtQNdEhJ4H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "cd359fa0-49ab-4e40-fa20-81c4424a619f"
      },
      "source": [
        "lr, num_epochs = 0.001, 5\n",
        "trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr})\n",
        "loss = gloss.SoftmaxCrossEntropyLoss()\n",
        "d2l.train(train_iter, test_iter, net, loss, trainer, ctx, num_epochs)"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training on [gpu(0)]\n",
            "epoch 1, loss 0.6080, train acc 0.716, test acc 0.832, time 19.3 sec\n",
            "epoch 2, loss 0.3614, train acc 0.839, test acc 0.854, time 18.9 sec\n",
            "epoch 3, loss 0.2665, train acc 0.891, test acc 0.821, time 18.9 sec\n",
            "epoch 4, loss 0.1764, train acc 0.933, test acc 0.862, time 18.9 sec\n",
            "epoch 5, loss 0.1024, train acc 0.965, test acc 0.865, time 18.9 sec\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pk_K-vcfhg3g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "018f6409-b427-4550-8473-daa342733424"
      },
      "source": [
        "d2l.predict_sentiment(net, vocab, ['this', 'movie', 'is', 'so', 'great'])"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'positive'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SCCg7yyAiOSv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1ff9c3c7-296c-46fd-f982-603d855d2427"
      },
      "source": [
        "d2l.predict_sentiment(net, vocab, ['this', 'movie', 'is', 'so', 'bad'])"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'negative'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    }
  ]
}